<!DOCTYPE html>
<html lang="zh-CN">
<head>
  <meta charset="UTF-8">
<meta name="viewport" content="width=device-width, initial-scale=1, maximum-scale=2">
<meta name="theme-color" content="#222">
<meta name="generator" content="Hexo 4.2.0">
  <link rel="apple-touch-icon" sizes="180x180" href="/images/favicon.png">
  <link rel="icon" type="image/png" sizes="32x32" href="/images/favicon.png">
  <link rel="icon" type="image/png" sizes="16x16" href="/images/favicon.png">
  <link rel="mask-icon" href="/images/favicon.png" color="#222">

<link rel="stylesheet" href="/css/main.css">


<link rel="stylesheet" href="/lib/font-awesome/css/font-awesome.min.css">
  <link rel="stylesheet" href="//cdn.jsdelivr.net/gh/fancyapps/fancybox@3/dist/jquery.fancybox.min.css">


<script id="hexo-configurations">
  var NexT = window.NexT || {};
  var CONFIG = {
    hostname: new URL('https://codz-me.vercel.app').hostname,
    root: '/',
    scheme: 'Muse',
    version: '7.7.1',
    exturl: false,
    sidebar: {"position":"right","display":"post","padding":18,"offset":12,"onmobile":false},
    copycode: {"enable":false,"show_result":false,"style":null},
    back2top: {"enable":true,"sidebar":false,"scrollpercent":false},
    bookmark: {"enable":false,"color":"#222","save":"auto"},
    fancybox: true,
    mediumzoom: false,
    lazyload: false,
    pangu: false,
    comments: {"style":"tabs","active":null,"storage":true,"lazyload":false,"nav":null},
    algolia: {
      appID: '',
      apiKey: '',
      indexName: '',
      hits: {"per_page":10},
      labels: {"input_placeholder":"Search for Posts","hits_empty":"We didn't find any results for the search: ${query}","hits_stats":"${hits} results found in ${time} ms"}
    },
    localsearch: {"enable":false,"trigger":"auto","top_n_per_article":1,"unescape":false,"preload":false},
    path: '',
    motion: {"enable":true,"async":false,"transition":{"post_block":"fadeIn","post_header":"slideDownIn","post_body":"slideDownIn","coll_header":"slideLeftIn","sidebar":"slideUpIn"}}
  };
</script>

  <meta name="description" content="Install pssh, to batch config spark slaves1234567wget https:&#x2F;&#x2F;storage.googleapis.com&#x2F;google-code-archive-downloads&#x2F;v2&#x2F;code.google.coms&#x2F;parallel-ssh&#x2F;pssh-2.3.1.tar.gztar -xzvf pssh-2.3.1.tar.gzcd pssh-">
<meta property="og:type" content="article">
<meta property="og:title" content="Build Spark Cluster">
<meta property="og:url" content="https://codz-me.vercel.app/2018/05/07/build-spark-cluster/index.html">
<meta property="og:site_name" content="Code Is Poetry">
<meta property="og:description" content="Install pssh, to batch config spark slaves1234567wget https:&#x2F;&#x2F;storage.googleapis.com&#x2F;google-code-archive-downloads&#x2F;v2&#x2F;code.google.coms&#x2F;parallel-ssh&#x2F;pssh-2.3.1.tar.gztar -xzvf pssh-2.3.1.tar.gzcd pssh-">
<meta property="og:locale" content="zh_CN">
<meta property="og:image" content="https://codz-me.vercel.app/2018/05/07/build-spark-cluster/spark-worker-ui-hostname.png">
<meta property="article:published_time" content="2018-05-07T09:47:03.000Z">
<meta property="article:modified_time" content="2018-06-09T07:00:52.000Z">
<meta property="article:author" content="laudukang">
<meta property="article:tag" content="spark">
<meta name="twitter:card" content="summary">
<meta name="twitter:image" content="https://codz-me.vercel.app/2018/05/07/build-spark-cluster/spark-worker-ui-hostname.png">

<link rel="canonical" href="https://codz-me.vercel.app/2018/05/07/build-spark-cluster/">


<script id="page-configurations">
  // https://hexo.io/docs/variables.html
  CONFIG.page = {
    sidebar: "",
    isHome: false,
    isPost: true
  };
</script>

  <title>Build Spark Cluster | Code Is Poetry</title>
  






  <noscript>
  <style>
  .use-motion .brand,
  .use-motion .menu-item,
  .sidebar-inner,
  .use-motion .post-block,
  .use-motion .pagination,
  .use-motion .comments,
  .use-motion .post-header,
  .use-motion .post-body,
  .use-motion .collection-header { opacity: initial; }

  .use-motion .site-title,
  .use-motion .site-subtitle {
    opacity: initial;
    top: initial;
  }

  .use-motion .logo-line-before i { left: initial; }
  .use-motion .logo-line-after i { right: initial; }
  </style>
</noscript>

</head>

<body itemscope itemtype="http://schema.org/WebPage">
  <div class="container use-motion">
    <div class="headband"></div>

    <header class="header" itemscope itemtype="http://schema.org/WPHeader">
      <div class="header-inner"><div class="site-brand-container">
  <div class="site-meta">

    <div>
      <a href="/" class="brand" rel="start">
        <span class="logo-line-before"><i></i></span>
        <span class="site-title">Code Is Poetry</span>
        <span class="logo-line-after"><i></i></span>
      </a>
    </div>
  </div>

  <div class="site-nav-toggle">
    <div class="toggle" aria-label="切换导航栏">
      <span class="toggle-line toggle-line-first"></span>
      <span class="toggle-line toggle-line-middle"></span>
      <span class="toggle-line toggle-line-last"></span>
    </div>
  </div>
</div>


<nav class="site-nav">
  
  <ul id="menu" class="menu">
        <li class="menu-item menu-item-home">

    <a href="/" rel="section"><i class="fa fa-fw fa-home"></i>首页</a>

  </li>
        <li class="menu-item menu-item-about">

    <a href="/about/" rel="section"><i class="fa fa-fw fa-user"></i>关于</a>

  </li>
        <li class="menu-item menu-item-tags">

    <a href="/tags/" rel="section"><i class="fa fa-fw fa-tags"></i>标签</a>

  </li>
        <li class="menu-item menu-item-categories">

    <a href="/categories/" rel="section"><i class="fa fa-fw fa-th"></i>分类</a>

  </li>
        <li class="menu-item menu-item-archives">

    <a href="/archives/" rel="section"><i class="fa fa-fw fa-archive"></i>归档</a>

  </li>
        <li class="menu-item menu-item-commonweal">

    <a href="/404/" rel="section"><i class="fa fa-fw fa-heartbeat"></i>公益 404</a>

  </li>
  </ul>

</nav>
</div>
    </header>

    
  <div class="back-to-top">
    <i class="fa fa-arrow-up"></i>
    <span>0%</span>
  </div>
  <div class="reading-progress-bar"></div>


    <main class="main">
      <div class="main-inner">
        <div class="content-wrap">
          

          <div class="content">
            

  <div class="posts-expand">
      
  
  
  <article itemscope itemtype="http://schema.org/Article" class="post-block " lang="zh-CN">
    <link itemprop="mainEntityOfPage" href="https://codz-me.vercel.app/2018/05/07/build-spark-cluster/">

    <span hidden itemprop="author" itemscope itemtype="http://schema.org/Person">
      <meta itemprop="image" content="/images/head.png">
      <meta itemprop="name" content="laudukang">
      <meta itemprop="description" content="Bug Not Found">
    </span>

    <span hidden itemprop="publisher" itemscope itemtype="http://schema.org/Organization">
      <meta itemprop="name" content="Code Is Poetry">
    </span>
      <header class="post-header">
        <h1 class="post-title" itemprop="name headline">
          Build Spark Cluster
        </h1>

        <div class="post-meta">
              <span class="post-meta-item">
                <span class="post-meta-item-icon">
                  <i class="fa fa-calendar-check-o"></i>
                </span>
                <span class="post-meta-item-text">更新于</span>
                <time title="修改时间：2018-06-09 15:00:52 15:00:52" itemprop="dateModified" datetime="2018-06-09T15:00:52+08:00">2018-06-09 15:00:52</time>
              </span>
            <span class="post-meta-item">
              <span class="post-meta-item-icon">
                <i class="fa fa-folder-o"></i>
              </span>
              <span class="post-meta-item-text">分类于</span>
                <span itemprop="about" itemscope itemtype="http://schema.org/Thing">
                  <a href="/categories/Linux/" itemprop="url" rel="index">
                    <span itemprop="name">Linux</span>
                  </a>
                </span>
            </span>

          
            <span id="/2018/05/07/build-spark-cluster/" class="post-meta-item leancloud_visitors" data-flag-title="Build Spark Cluster" title="阅读次数">
                <span hidden class="leancloud-visitors-count"></span>
            </span>

        </div>
      </header>

    
    
    
    <div class="post-body" itemprop="articleBody">

      
        <h2 id="Install-pssh-to-batch-config-spark-slaves"><a href="#Install-pssh-to-batch-config-spark-slaves" class="headerlink" title="Install pssh, to batch config spark slaves"></a>Install pssh, to batch config spark slaves</h2><figure class="highlight shell"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br></pre></td><td class="code"><pre><span class="line">wget https://storage.googleapis.com/google-code-archive-downloads/v2/code.google.coms/parallel-ssh/pssh-2.3.1.tar.gz</span><br><span class="line"></span><br><span class="line">tar -xzvf pssh-2.3.1.tar.gz</span><br><span class="line"></span><br><span class="line">cd pssh-2.3.1</span><br><span class="line"></span><br><span class="line">python setup.py install</span><br></pre></td></tr></table></figure>

<p>使用参见：</p>
<ul>
<li><p><a href="http://www.cnblogs.com/lurenjiashuo/p/pssh.html" target="_blank" rel="noopener">使用PSSH批量操作Linux服务器</a></p>
</li>
<li><p><a href="https://tonydeng.github.io/2014/12/08/pssh/" target="_blank" rel="noopener">PSSH基本使用介绍</a></p>
</li>
</ul>
<h2 id="System-config-in-master"><a href="#System-config-in-master" class="headerlink" title="System config in master"></a>System config in master</h2><p>/work/hosts</p>
<figure class="highlight plain"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br></pre></td><td class="code"><pre><span class="line">root@192.168.4.210</span><br><span class="line">root@192.168.4.211</span><br><span class="line">root@192.168.4.212</span><br><span class="line">root@192.168.4.213</span><br><span class="line">root@192.168.4.214</span><br><span class="line">root@192.168.4.215</span><br><span class="line">root@192.168.4.216</span><br><span class="line">root@192.168.4.217</span><br></pre></td></tr></table></figure>

<p>/etc/hosts</p>
<figure class="highlight plain"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br><span class="line">9</span><br><span class="line">10</span><br><span class="line">11</span><br><span class="line">12</span><br><span class="line">13</span><br><span class="line">14</span><br><span class="line">15</span><br></pre></td><td class="code"><pre><span class="line">127.0.0.1       localhost</span><br><span class="line">127.0.1.1       ubuntu</span><br><span class="line"></span><br><span class="line"># The following lines are desirable for IPv6 capable hosts</span><br><span class="line">::1     localhost ip6-localhost ip6-loopback</span><br><span class="line">ff02::1 ip6-allnodes</span><br><span class="line">ff02::2 ip6-allrouters</span><br><span class="line"></span><br><span class="line">192.168.4.211 n1</span><br><span class="line">192.168.4.212 n2</span><br><span class="line">192.168.4.213 n3</span><br><span class="line">192.168.4.214 n4</span><br><span class="line">192.168.4.215 n5</span><br><span class="line">192.168.4.216 n6</span><br><span class="line">192.168.4.217 n7</span><br></pre></td></tr></table></figure>

<p>/etc/environment</p>
<figure class="highlight plain"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br><span class="line">9</span><br><span class="line">10</span><br></pre></td><td class="code"><pre><span class="line">LANGUAGE&#x3D;&quot;zh_CN:zh:en_US:en&quot;</span><br><span class="line">LANG&#x3D;zh_CN.GBK</span><br><span class="line">SPARK_HOME&#x3D;&#x2F;work&#x2F;spark</span><br><span class="line">SCALA_HOME&#x3D;&#x2F;opt&#x2F;scala</span><br><span class="line">JAVA_HOME&#x3D;&#x2F;opt&#x2F;java</span><br><span class="line">J2SDKDIR&#x3D;&#x2F;opt&#x2F;java</span><br><span class="line">J2REDIR&#x3D;&#x2F;opt&#x2F;java&#x2F;jre</span><br><span class="line">DERBY_HOME&#x3D;&#x2F;opt&#x2F;java&#x2F;db</span><br><span class="line">PATH&#x3D;&quot;&#x2F;usr&#x2F;local&#x2F;sbin:&#x2F;usr&#x2F;local&#x2F;bin:&#x2F;usr&#x2F;sbin:&#x2F;usr&#x2F;bin:&#x2F;sbin:&#x2F;bin:&#x2F;usr&#x2F;games:&#x2F;usr&#x2F;local&#x2F;games:&#x2F;opt&#x2F;java&#x2F;bin:&#x2F;opt&#x2F;scala&#x2F;bin:&#x2F;work&#x2F;spark&#x2F;bin&quot;</span><br><span class="line">CLASSPATH&#x3D;..:&#x2F;opt&#x2F;java&#x2F;lib:&#x2F;opt&#x2F;java&#x2F;jre&#x2F;lib:&#x2F;opt&#x2F;scala&#x2F;lib:&#x2F;work&#x2F;spark&#x2F;jars</span><br></pre></td></tr></table></figure>

<p>/work/spark/conf/spark-env.sh,update <code>ens3</code> to your network interface</p>
<figure class="highlight shell"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br></pre></td><td class="code"><pre><span class="line">export SPARK_LOCAL_IP=$(ifconfig ens3 | grep "inet addr:" | awk '&#123;print $2&#125;' | cut -c 6-)</span><br><span class="line">export SPARK_MASTER_HOST=n1</span><br><span class="line">export SPARK_MASTER_PORT=7077</span><br><span class="line">export SPARK_MASTER_WEBUI_PORT=8000</span><br><span class="line">export SPARK_HISTORY_OPTS="-Dspark.history.fs.logDirectory=/work/spark/logs-event"</span><br><span class="line"><span class="meta">#</span><span class="bash"><span class="built_in">export</span> SPARK_EXECUTOR_CORES=1</span></span><br><span class="line"><span class="meta">#</span><span class="bash"><span class="built_in">export</span> SPARK_EXECUTOR_MEMORY=512M</span></span><br><span class="line"><span class="meta">#</span><span class="bash"><span class="built_in">export</span> SPARK_WORKER_INSTANCES=4</span></span><br></pre></td></tr></table></figure>

<p>update_hostname.sh, update default hostname <code>ubuntu</code> to <code>n{machine ip last number}</code></p>
<figure class="highlight shell"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br><span class="line">9</span><br></pre></td><td class="code"><pre><span class="line"><span class="meta">#</span><span class="bash">!/usr/bin/env bash</span></span><br><span class="line">NODE_LOCAL_IP=$(ifconfig ens3 | grep "inet addr:" | awk '&#123;print $2&#125;' | cut -c 6-)</span><br><span class="line">NEW_HOSTNAME="n$&#123;NODE_LOCAL_IP:12&#125;"</span><br><span class="line"></span><br><span class="line"><span class="meta">#</span><span class="bash"><span class="built_in">echo</span> <span class="variable">$NEW_HOSTNAME</span> &gt; /proc/sys/kernel/hostname</span></span><br><span class="line"><span class="meta">#</span><span class="bash"><span class="built_in">echo</span> <span class="variable">$NEW_HOSTNAME</span> &gt; /etc/hostname</span></span><br><span class="line"></span><br><span class="line">sed -i 's/127.0.1.1.*/127.0.1.1\t'"$NEW_HOSTNAME"'/g' /etc/hosts</span><br><span class="line">hostnamectl set-hostname $NEW_HOSTNAME</span><br></pre></td></tr></table></figure>

<h2 id="Init-master-and-slaves-in-master"><a href="#Init-master-and-slaves-in-master" class="headerlink" title="Init master and slaves in master"></a>Init master and slaves in master</h2><figure class="highlight shell"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br><span class="line">9</span><br><span class="line">10</span><br><span class="line">11</span><br><span class="line">12</span><br><span class="line">13</span><br><span class="line">14</span><br><span class="line">15</span><br><span class="line">16</span><br><span class="line">17</span><br><span class="line">18</span><br><span class="line">19</span><br><span class="line">20</span><br><span class="line">21</span><br><span class="line">22</span><br><span class="line">23</span><br><span class="line">24</span><br><span class="line">25</span><br><span class="line">26</span><br><span class="line">27</span><br><span class="line">28</span><br><span class="line">29</span><br><span class="line">30</span><br><span class="line">31</span><br><span class="line">32</span><br><span class="line">33</span><br><span class="line">34</span><br><span class="line">35</span><br><span class="line">36</span><br><span class="line">37</span><br><span class="line">38</span><br><span class="line">39</span><br><span class="line">40</span><br><span class="line">41</span><br><span class="line">42</span><br></pre></td><td class="code"><pre><span class="line">pssh -h hosts "apt update"</span><br><span class="line">pssh -h hosts "apt install htop -y"</span><br><span class="line"></span><br><span class="line"><span class="meta">#</span><span class="bash">zsh init</span></span><br><span class="line">pssh -h hosts "apt install zsh -y"</span><br><span class="line">pssh -h hosts 'sh -c "$(curl -fsSL https://raw.github.com/robbyrussell/oh-my-zsh/master/tools/install.sh)";'</span><br><span class="line">pssh -h hosts chsh -s $(which zsh)</span><br><span class="line"></span><br><span class="line"><span class="meta">#</span><span class="bash">update dns resolve</span></span><br><span class="line">pscp -h hosts /etc/resolvconf/resolv.conf.d/base /etc/resolvconf/resolv.conf.d/base</span><br><span class="line">pssh -h hosts "resolvconf -u"</span><br><span class="line"></span><br><span class="line"><span class="meta">#</span><span class="bash">host update</span></span><br><span class="line">pscp -h hosts /etc/hosts /etc</span><br><span class="line"></span><br><span class="line"><span class="meta">#</span><span class="bash">create work directory</span></span><br><span class="line">pssh -h hosts "mkdir /work"</span><br><span class="line"></span><br><span class="line"><span class="meta">#</span><span class="bash">copy from master: 192.168.4.210</span></span><br><span class="line"><span class="meta">#</span><span class="bash">plesae remove 192.168.4.210 <span class="keyword">in</span> hosts</span></span><br><span class="line"></span><br><span class="line">tar -zxvf jdk-8u172-linux-x64.tar.gz -C /opt</span><br><span class="line">tar -zxvf scala-2.11.12.tgz -C /opt</span><br><span class="line">tar -zxvf scala-2.11.12.tgz -C /opt</span><br><span class="line">mv /opt/jdk-8u172-linux-x64 /opt/java</span><br><span class="line">mv /opt/scala-2.11.12 /opt/scala</span><br><span class="line"></span><br><span class="line">tar -zxvf spark-2.3.0-bin-hadoop2.7.tgz -C /work</span><br><span class="line">mv spark-2.3.0-bin-hadoop2.7 spark</span><br><span class="line"></span><br><span class="line"><span class="meta">#</span><span class="bash">copy java, scala</span></span><br><span class="line">pscp -r -h hosts /opt/* /opt/</span><br><span class="line"><span class="meta">#</span><span class="bash">copy spark</span></span><br><span class="line">pscp -r -h hosts /work/spark /work</span><br><span class="line"></span><br><span class="line"><span class="meta">#</span><span class="bash">update env variables</span></span><br><span class="line">pscp -h hosts /etc/environment /etc</span><br><span class="line">pssh -h hosts "source /etc/environment"</span><br><span class="line"></span><br><span class="line"><span class="meta">#</span><span class="bash">update hostname</span></span><br><span class="line">pscp -h hosts update_hostname.sh /tmp/</span><br><span class="line">pssh -h hosts "./tmp/update_hostname.sh"</span><br></pre></td></tr></table></figure>


<h2 id="Hadoop-config-in-master"><a href="#Hadoop-config-in-master" class="headerlink" title="Hadoop config in master"></a>Hadoop config in master</h2><p>此处引入<code>hadoop</code>是为了slaves从<code>hadoop</code>拉去资源启动app；</p>
<p>当然，也可以复制jar到各slaves相同的路径启动；</p>
<p>后面发现，每次重启app，spark都会从<code>hadoop</code>全量拉取一遍资源到<code>spark/work</code>目录。</p>
<p>core-site.xml</p>
<figure class="highlight xml"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br><span class="line">9</span><br><span class="line">10</span><br><span class="line">11</span><br><span class="line">12</span><br><span class="line">13</span><br><span class="line">14</span><br></pre></td><td class="code"><pre><span class="line"><span class="meta">&lt;?xml version="1.0" encoding="UTF-8"?&gt;</span></span><br><span class="line"><span class="meta">&lt;?xml-stylesheet type="text/xsl" href="configuration.xsl"?&gt;</span></span><br><span class="line"></span><br><span class="line"><span class="tag">&lt;<span class="name">configuration</span>&gt;</span></span><br><span class="line">        <span class="tag">&lt;<span class="name">property</span>&gt;</span></span><br><span class="line">              <span class="tag">&lt;<span class="name">name</span>&gt;</span>hadoop.tmp.dir<span class="tag">&lt;/<span class="name">name</span>&gt;</span></span><br><span class="line">              <span class="tag">&lt;<span class="name">value</span>&gt;</span>file:/work/hadoop/tmp<span class="tag">&lt;/<span class="name">value</span>&gt;</span></span><br><span class="line">              <span class="tag">&lt;<span class="name">description</span>&gt;</span>Abase for other temporary directories.<span class="tag">&lt;/<span class="name">description</span>&gt;</span></span><br><span class="line">        <span class="tag">&lt;/<span class="name">property</span>&gt;</span></span><br><span class="line">        <span class="tag">&lt;<span class="name">property</span>&gt;</span></span><br><span class="line">              <span class="tag">&lt;<span class="name">name</span>&gt;</span>fs.defaultFS<span class="tag">&lt;/<span class="name">name</span>&gt;</span></span><br><span class="line">              <span class="tag">&lt;<span class="name">value</span>&gt;</span>hdfs://n0:9820<span class="tag">&lt;/<span class="name">value</span>&gt;</span></span><br><span class="line">        <span class="tag">&lt;/<span class="name">property</span>&gt;</span></span><br><span class="line"><span class="tag">&lt;/<span class="name">configuration</span>&gt;</span></span><br></pre></td></tr></table></figure>

<p>hdfs-site.xml</p>
<figure class="highlight xml"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br><span class="line">9</span><br><span class="line">10</span><br><span class="line">11</span><br><span class="line">12</span><br><span class="line">13</span><br><span class="line">14</span><br><span class="line">15</span><br><span class="line">16</span><br><span class="line">17</span><br><span class="line">18</span><br><span class="line">19</span><br><span class="line">20</span><br><span class="line">21</span><br><span class="line">22</span><br><span class="line">23</span><br><span class="line">24</span><br><span class="line">25</span><br><span class="line">26</span><br><span class="line">27</span><br><span class="line">28</span><br><span class="line">29</span><br><span class="line">30</span><br><span class="line">31</span><br><span class="line">32</span><br><span class="line">33</span><br><span class="line">34</span><br><span class="line">35</span><br><span class="line">36</span><br><span class="line">37</span><br><span class="line">38</span><br><span class="line">39</span><br><span class="line">40</span><br><span class="line">41</span><br><span class="line">42</span><br><span class="line">43</span><br><span class="line">44</span><br><span class="line">45</span><br><span class="line">46</span><br><span class="line">47</span><br><span class="line">48</span><br><span class="line">49</span><br></pre></td><td class="code"><pre><span class="line"><span class="meta">&lt;?xml version="1.0" encoding="UTF-8"?&gt;</span></span><br><span class="line"><span class="meta">&lt;?xml-stylesheet type="text/xsl" href="configuration.xsl"?&gt;</span></span><br><span class="line"></span><br><span class="line"><span class="tag">&lt;<span class="name">configuration</span>&gt;</span></span><br><span class="line">        <span class="tag">&lt;<span class="name">property</span>&gt;</span></span><br><span class="line">              <span class="tag">&lt;<span class="name">name</span>&gt;</span>dfs.replication<span class="tag">&lt;/<span class="name">name</span>&gt;</span></span><br><span class="line">              <span class="tag">&lt;<span class="name">value</span>&gt;</span>1<span class="tag">&lt;/<span class="name">value</span>&gt;</span></span><br><span class="line">        <span class="tag">&lt;/<span class="name">property</span>&gt;</span></span><br><span class="line">        <span class="tag">&lt;<span class="name">property</span>&gt;</span></span><br><span class="line">              <span class="tag">&lt;<span class="name">name</span>&gt;</span>dfs.namenode.name.dir<span class="tag">&lt;/<span class="name">name</span>&gt;</span></span><br><span class="line">              <span class="tag">&lt;<span class="name">value</span>&gt;</span>file:/work/hadoop/tmp/dfs/namenode<span class="tag">&lt;/<span class="name">value</span>&gt;</span></span><br><span class="line">        <span class="tag">&lt;/<span class="name">property</span>&gt;</span></span><br><span class="line">        <span class="tag">&lt;<span class="name">property</span>&gt;</span></span><br><span class="line">              <span class="tag">&lt;<span class="name">name</span>&gt;</span>dfs.datanode.data.dir<span class="tag">&lt;/<span class="name">name</span>&gt;</span></span><br><span class="line">              <span class="tag">&lt;<span class="name">value</span>&gt;</span>file:/work/hadoop/tmp/dfs/datanode<span class="tag">&lt;/<span class="name">value</span>&gt;</span></span><br><span class="line">        <span class="tag">&lt;/<span class="name">property</span>&gt;</span></span><br><span class="line">        <span class="tag">&lt;<span class="name">property</span>&gt;</span></span><br><span class="line">            <span class="tag">&lt;<span class="name">name</span>&gt;</span>dfs.http.address<span class="tag">&lt;/<span class="name">name</span>&gt;</span></span><br><span class="line">            <span class="tag">&lt;<span class="name">value</span>&gt;</span>n0:9870<span class="tag">&lt;/<span class="name">value</span>&gt;</span></span><br><span class="line">        <span class="tag">&lt;/<span class="name">property</span>&gt;</span></span><br><span class="line">        <span class="tag">&lt;<span class="name">property</span>&gt;</span></span><br><span class="line">            <span class="tag">&lt;<span class="name">name</span>&gt;</span>dfs.permissions.enabled<span class="tag">&lt;/<span class="name">name</span>&gt;</span></span><br><span class="line">            <span class="tag">&lt;<span class="name">value</span>&gt;</span>false<span class="tag">&lt;/<span class="name">value</span>&gt;</span></span><br><span class="line">        <span class="tag">&lt;/<span class="name">property</span>&gt;</span></span><br><span class="line">        <span class="tag">&lt;<span class="name">property</span>&gt;</span></span><br><span class="line">            <span class="tag">&lt;<span class="name">name</span>&gt;</span>dfs.permissions.superusergroup<span class="tag">&lt;/<span class="name">name</span>&gt;</span></span><br><span class="line">            <span class="tag">&lt;<span class="name">value</span>&gt;</span>hadoop<span class="tag">&lt;/<span class="name">value</span>&gt;</span></span><br><span class="line">        <span class="tag">&lt;/<span class="name">property</span>&gt;</span></span><br><span class="line">        <span class="tag">&lt;<span class="name">property</span>&gt;</span></span><br><span class="line">            <span class="tag">&lt;<span class="name">name</span>&gt;</span>hadoop.http.staticuser.user<span class="tag">&lt;/<span class="name">name</span>&gt;</span></span><br><span class="line">            <span class="tag">&lt;<span class="name">value</span>&gt;</span>hadoop<span class="tag">&lt;/<span class="name">value</span>&gt;</span></span><br><span class="line">        <span class="tag">&lt;/<span class="name">property</span>&gt;</span></span><br><span class="line">        <span class="tag">&lt;<span class="name">property</span>&gt;</span></span><br><span class="line">            <span class="tag">&lt;<span class="name">name</span>&gt;</span>dfs.secondary.http.address<span class="tag">&lt;/<span class="name">name</span>&gt;</span></span><br><span class="line">            <span class="tag">&lt;<span class="name">value</span>&gt;</span>n0:9868<span class="tag">&lt;/<span class="name">value</span>&gt;</span></span><br><span class="line">        <span class="tag">&lt;/<span class="name">property</span>&gt;</span></span><br><span class="line">        <span class="tag">&lt;<span class="name">property</span>&gt;</span></span><br><span class="line">            <span class="tag">&lt;<span class="name">name</span>&gt;</span>dfs.datanode.http.address<span class="tag">&lt;/<span class="name">name</span>&gt;</span></span><br><span class="line">            <span class="tag">&lt;<span class="name">value</span>&gt;</span>n0:9864<span class="tag">&lt;/<span class="name">value</span>&gt;</span></span><br><span class="line">        <span class="tag">&lt;/<span class="name">property</span>&gt;</span></span><br><span class="line">        <span class="tag">&lt;<span class="name">property</span>&gt;</span></span><br><span class="line">            <span class="tag">&lt;<span class="name">name</span>&gt;</span>dfs.datanode.address<span class="tag">&lt;/<span class="name">name</span>&gt;</span></span><br><span class="line">            <span class="tag">&lt;<span class="name">value</span>&gt;</span>n0:9866<span class="tag">&lt;/<span class="name">value</span>&gt;</span></span><br><span class="line">        <span class="tag">&lt;/<span class="name">property</span>&gt;</span></span><br><span class="line">        <span class="tag">&lt;<span class="name">property</span>&gt;</span></span><br><span class="line">            <span class="tag">&lt;<span class="name">name</span>&gt;</span>dfs.datanode.hostname<span class="tag">&lt;/<span class="name">name</span>&gt;</span></span><br><span class="line">            <span class="tag">&lt;<span class="name">value</span>&gt;</span>n0<span class="tag">&lt;/<span class="name">value</span>&gt;</span></span><br><span class="line">        <span class="tag">&lt;/<span class="name">property</span>&gt;</span></span><br><span class="line"><span class="tag">&lt;/<span class="name">configuration</span>&gt;</span></span><br></pre></td></tr></table></figure>

<p>init <code>hadoop</code></p>
<figure class="highlight shell"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br><span class="line">9</span><br></pre></td><td class="code"><pre><span class="line"><span class="meta">#</span><span class="bash">create hadoop user <span class="keyword">in</span> master</span></span><br><span class="line">sudo useradd -m hadoop -s /bin/bash</span><br><span class="line"></span><br><span class="line"><span class="meta">#</span><span class="bash">/work/hadoop/bin</span></span><br><span class="line">./hdfs dfs -mkdir -p /sparkHistoryLogs</span><br><span class="line">./hdfs dfs -mkdir -p /eventLogs</span><br><span class="line">./hdfs dfs -mkdir -p /spark</span><br><span class="line"></span><br><span class="line"><span class="meta">#</span><span class="bash">./hdfs dfs -rm -R /spark/app/*</span></span><br></pre></td></tr></table></figure>

<p>copy_app_resouces_to_hadoop.sh, run as <code>hadoop</code> user</p>
<figure class="highlight shell"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br><span class="line">9</span><br><span class="line">10</span><br></pre></td><td class="code"><pre><span class="line"><span class="meta">#</span><span class="bash">!/bin/bash</span></span><br><span class="line">cd /work/hadoop/bin</span><br><span class="line"><span class="meta">#</span><span class="bash">./hdfs dfs -mkdir -p /spark</span></span><br><span class="line">./hdfs dfs -rm -R /spark/app/*</span><br><span class="line">./hdfs dfs -copyFromLocal -f /work/spark/app/log4j.properties /spark/app</span><br><span class="line"><span class="meta">#</span><span class="bash">spark config</span></span><br><span class="line">./hdfs dfs -copyFromLocal -f /work/spark/app/default.conf /spark/app</span><br><span class="line"><span class="meta">#</span><span class="bash">app dependencies</span></span><br><span class="line">./hdfs dfs -copyFromLocal -f /work/spark/app/lib /spark/app</span><br><span class="line">./hdfs dfs -copyFromLocal -f /work/spark/app/node-quality-streaming-0.0.1-SNAPSHOT.jar /spark/app</span><br></pre></td></tr></table></figure>

<h2 id="Exception-solutions"><a href="#Exception-solutions" class="headerlink" title="Exception solutions"></a>Exception solutions</h2><ul>
<li><p>fix spark WorkWebUI hostname(logs) 指向master机器hostname</p>
<p>看源码，在<code>spark-env.sh</code>中指定<code>SPARK_LOCAL_HOSTNAME</code>并没起作用，</p>
<p>解决方案：设置<code>SPARK_PUBLIC_DNS</code>参数后，worker webui中的跳转链接正常了。</p>
<p><code>SPARK_PUBLIC_DNS</code>→<code>publicHostName</code>我也是服了，</p>
<p>如下图，原先<code>stdout</code>的链接的的主机为<code>n0</code>，<code>n0</code>为master所在的机器：</p>
<img src="/2018/05/07/build-spark-cluster/spark-worker-ui-hostname.png" alt="spark-worker-ui-hostname.png" title="">

<p>源码参考</p>
<p><a href="https://github.com/apache/spark/blob/master/core/src/main/scala/org/apache/spark/ui/WebUI.scala" target="_blank" rel="noopener">core/src/main/scala/org/apache/spark/ui/WebUI.scala</a></p>
<figure class="highlight scala"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br></pre></td><td class="code"><pre><span class="line"><span class="keyword">protected</span> <span class="keyword">val</span> publicHostName = <span class="type">Option</span>(conf.getenv(<span class="string">"SPARK_PUBLIC_DNS"</span>)).getOrElse(</span><br><span class="line">    conf.get(<span class="type">DRIVER_HOST_ADDRESS</span>))</span><br><span class="line"></span><br><span class="line"><span class="comment">/** Return the url of web interface. Only valid after bind(). */</span></span><br><span class="line"><span class="function"><span class="keyword">def</span> <span class="title">webUrl</span></span>: <span class="type">String</span> = <span class="string">s"http://<span class="subst">$publicHostName</span>:<span class="subst">$boundPort</span>"</span></span><br></pre></td></tr></table></figure>

<p><a href="https://github.com/apache/spark/blob/master/core/src/main/scala/org/apache/spark/util/Utils.scala" target="_blank" rel="noopener">core/src/main/scala/org/apache/spark/util/Utils.scala</a></p>
<figure class="highlight scala"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br><span class="line">9</span><br></pre></td><td class="code"><pre><span class="line"><span class="keyword">private</span> <span class="keyword">lazy</span> <span class="keyword">val</span> localIpAddress: <span class="type">InetAddress</span> = findLocalInetAddress()</span><br><span class="line"><span class="keyword">private</span> <span class="keyword">var</span> customHostname: <span class="type">Option</span>[<span class="type">String</span>] = sys.env.get(<span class="string">"SPARK_LOCAL_HOSTNAME"</span>)</span><br><span class="line"></span><br><span class="line"><span class="comment">/**</span></span><br><span class="line"><span class="comment"> * Get the local machine's URI.</span></span><br><span class="line"><span class="comment"> */</span></span><br><span class="line"><span class="function"><span class="keyword">def</span> <span class="title">localHostNameForURI</span></span>(): <span class="type">String</span> = &#123;</span><br><span class="line">  customHostname.getOrElse(<span class="type">InetAddresses</span>.toUriString(localIpAddress))</span><br><span class="line">&#125;</span><br></pre></td></tr></table></figure>


</li>
</ul>
<ul>
<li><p>spark ConsumerRecord NotSerializableException bug</p>
<figure class="highlight java"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br><span class="line">9</span><br><span class="line">10</span><br><span class="line">11</span><br><span class="line">12</span><br></pre></td><td class="code"><pre><span class="line">java.io.NotSerializableException: org.apache.kafka.clients.consumer.ConsumerRecord</span><br><span class="line">Serialization stack:</span><br><span class="line">    - <span class="function">object not <span class="title">serializable</span> <span class="params">(class: org.apache.kafka.clients.consumer.ConsumerRecord, value: ConsumerRecord(topic = hi2, partition = <span class="number">4</span>, offset = <span class="number">385</span>, CreateTime = <span class="number">1526369397516</span>, checksum = <span class="number">2122851237</span>, serialized key size = <span class="number">-1</span>, serialized value size = <span class="number">45</span>, key = <span class="keyword">null</span>, value = &#123;<span class="string">"date"</span>:<span class="number">1526369397516</span>,<span class="string">"message"</span>:<span class="string">"0hh2KcCH4j"</span>&#125;)</span>)</span></span><br><span class="line"><span class="function">    - element of <span class="title">array</span> <span class="params">(index: <span class="number">0</span>)</span></span></span><br><span class="line"><span class="function">    - <span class="title">array</span> <span class="params">(class [Lorg.apache.kafka.clients.consumer.ConsumerRecord;, size <span class="number">125</span>)</span></span></span><br><span class="line"><span class="function">    at org.apache.spark.serializer.SerializationDebugger$.<span class="title">improveException</span><span class="params">(SerializationDebugger.scala:<span class="number">40</span>)</span></span></span><br><span class="line"><span class="function">    at org.apache.spark.serializer.JavaSerializationStream.<span class="title">writeObject</span><span class="params">(JavaSerializer.scala:<span class="number">46</span>)</span></span></span><br><span class="line"><span class="function">    at org.apache.spark.serializer.JavaSerializerInstance.<span class="title">serialize</span><span class="params">(JavaSerializer.scala:<span class="number">100</span>)</span></span></span><br><span class="line"><span class="function">    at org.apache.spark.executor.Executor$TaskRunner.<span class="title">run</span><span class="params">(Executor.scala:<span class="number">393</span>)</span></span></span><br><span class="line"><span class="function">    at java.util.concurrent.ThreadPoolExecutor.<span class="title">runWorker</span><span class="params">(ThreadPoolExecutor.java:<span class="number">1149</span>)</span></span></span><br><span class="line"><span class="function">    at java.util.concurrent.ThreadPoolExecutor$Worker.<span class="title">run</span><span class="params">(ThreadPoolExecutor.java:<span class="number">624</span>)</span></span></span><br><span class="line"><span class="function">    at java.lang.Thread.<span class="title">run</span><span class="params">(Thread.java:<span class="number">748</span>)</span></span></span><br></pre></td></tr></table></figure>

<p>解决方案</p>
<p>set SparkConf</p>
<figure class="highlight java"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br></pre></td><td class="code"><pre><span class="line">sparkConf.set(<span class="string">"spark.serializer"</span>, <span class="string">"org.apache.spark.serializer.KryoSerializer"</span>);</span><br><span class="line">sparkConf.set(<span class="string">"spark.kryo.registrator"</span>, <span class="string">"me.codz.registrator.CunstomRegistrator"</span>);</span><br></pre></td></tr></table></figure>

<p>create CunstomRegistrator</p>
<figure class="highlight java"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br><span class="line">9</span><br><span class="line">10</span><br><span class="line">11</span><br><span class="line">12</span><br><span class="line">13</span><br></pre></td><td class="code"><pre><span class="line"><span class="keyword">package</span> me.codz.registrator;</span><br><span class="line"></span><br><span class="line"><span class="keyword">import</span> com.esotericsoftware.kryo.Kryo;</span><br><span class="line"><span class="keyword">import</span> org.apache.kafka.clients.consumer.ConsumerRecord;</span><br><span class="line"><span class="keyword">import</span> org.apache.spark.serializer.KryoRegistrator;</span><br><span class="line"></span><br><span class="line"><span class="keyword">public</span> <span class="class"><span class="keyword">class</span> <span class="title">CunstomRegistrator</span> <span class="keyword">implements</span> <span class="title">KryoRegistrator</span> </span>&#123;</span><br><span class="line"></span><br><span class="line">    <span class="meta">@Override</span></span><br><span class="line">    <span class="function"><span class="keyword">public</span> <span class="keyword">void</span> <span class="title">registerClasses</span><span class="params">(Kryo kryo)</span> </span>&#123;</span><br><span class="line">        kryo.register(ConsumerRecord<span class="class">.<span class="keyword">class</span>)</span>;</span><br><span class="line">    &#125;</span><br><span class="line">&#125;</span><br></pre></td></tr></table></figure>
</li>
<li><p>spark TaskContext.get() cause NullPointerException</p>
<figure class="highlight java"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br><span class="line">9</span><br><span class="line">10</span><br><span class="line">11</span><br><span class="line">12</span><br><span class="line">13</span><br></pre></td><td class="code"><pre><span class="line">stream.foreachRDD((VoidFunction2&lt;JavaRDD&lt;ConsumerRecord&lt;String, String&gt;&gt;, Time&gt;) (v1, v2) -&gt; &#123;</span><br><span class="line">           OffsetRange[] offsetRanges = ((HasOffsetRanges) v1.rdd()).offsetRanges();</span><br><span class="line"></span><br><span class="line">           List&lt;ConsumerRecord&lt;String, String&gt;&gt; consumerRecordList = CollectionTools.emptyWrapper(v1.collect());</span><br><span class="line">           consumerRecordList.forEach(consumerRecord -&gt; &#123;</span><br><span class="line">               TaskContext taskContext = TaskContext.get();</span><br><span class="line">               <span class="keyword">int</span> partitionId = taskContext.partitionId();</span><br><span class="line">               OffsetRange o = offsetRanges[partitionId];</span><br><span class="line"></span><br><span class="line">               <span class="comment">//...</span></span><br><span class="line">           &#125;);</span><br><span class="line"></span><br><span class="line">       &#125;);</span><br></pre></td></tr></table></figure>

<p>解决方案<br>using <code>foreachPartition</code></p>
<figure class="highlight java"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br><span class="line">9</span><br><span class="line">10</span><br><span class="line">11</span><br><span class="line">12</span><br><span class="line">13</span><br><span class="line">14</span><br></pre></td><td class="code"><pre><span class="line">v1.foreachPartition(consumerRecordIterator -&gt; &#123;</span><br><span class="line">                <span class="keyword">while</span> (consumerRecordIterator.hasNext()) &#123;</span><br><span class="line">                    ConsumerRecord&lt;String, String&gt; consumerRecord = consumerRecordIterator.next();</span><br><span class="line"></span><br><span class="line">                    <span class="comment">//...</span></span><br><span class="line">                &#125;</span><br><span class="line"></span><br><span class="line">                TaskContext taskContext = TaskContext.get();</span><br><span class="line">                <span class="keyword">int</span> partitionId = taskContext.partitionId();</span><br><span class="line">                OffsetRange offsetRange = offsetRanges[partitionId];</span><br><span class="line"></span><br><span class="line">                <span class="comment">//...</span></span><br><span class="line">            &#125;);</span><br><span class="line">        &#125;);</span><br></pre></td></tr></table></figure>
</li>
<li><p>Spark app connect kafka server by ip suspend</p>
<figure class="highlight plain"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br></pre></td><td class="code"><pre><span class="line">2018-06-07 18:40:16 [ForkJoinPool-1-worker-5] INFO :: [Consumer clientId&#x3D;consumer-1, groupId&#x3D;node-quality-streaming] Discovered group coordinator lau.cc:9092 (id: 2147483647 rack: null)</span><br><span class="line">2018-06-07 18:40:18 [ForkJoinPool-1-worker-5] INFO :: [Consumer clientId&#x3D;consumer-1, groupId&#x3D;node-quality-streaming] Group coordinator lau.cc:9092 (id: 2147483647 rack: null) is unavailable or invalid, will attempt rediscovery</span><br></pre></td></tr></table></figure>

<p>解决方案</p>
<p>vim kafka/config/server.properties</p>
<figure class="highlight plain"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br></pre></td><td class="code"><pre><span class="line">#add follow line</span><br><span class="line">advertised.host.name&#x3D;192.168.3.20</span><br></pre></td></tr></table></figure>

<p>kafka <code>advertised.host.name</code> DEPRECATED since 0.10.x, <a href="https://kafka.apache.org/0100/documentation.html#brokerconfigs" target="_blank" rel="noopener">0100 brokerconfigs</a></p>
<figure class="highlight plain"><table><tr><td class="gutter"><pre><span class="line">1</span><br></pre></td><td class="code"><pre><span class="line">DEPRECATED: only used when &#96;advertised.listeners&#96; or &#96;listeners&#96; are not set. Use &#96;advertised.listeners&#96; instead. Hostname to publish to ZooKeeper for clients to use. In IaaS environments, this may need to be different from the interface to which the broker binds. If this is not set, it will use the value for &#96;host.name&#96; if configured. Otherwise it will use the value returned from java.net.InetAddress.getCanonicalHostName().</span><br></pre></td></tr></table></figure>

<p>so follow config can also take effect</p>
<figure class="highlight plain"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br></pre></td><td class="code"><pre><span class="line">listeners&#x3D;PLAINTEXT:&#x2F;&#x2F;192.168.3.20:9092</span><br><span class="line">advertised.listeners&#x3D;PLAINTEXT:&#x2F;&#x2F;192.168.3.20:9092</span><br></pre></td></tr></table></figure>
</li>
<li><p>mark</p>
<figure class="highlight plain"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br></pre></td><td class="code"><pre><span class="line">java.lang.OutOfMemoryError: unable to create new native thread</span><br><span class="line"></span><br><span class="line">Offsets out of range with no configured reset policy for partitions</span><br></pre></td></tr></table></figure>

</li>
</ul>
<h2 id="Zookeeper-cluster-init"><a href="#Zookeeper-cluster-init" class="headerlink" title="Zookeeper cluster init"></a>Zookeeper cluster init</h2><p><a href="https://www.apache.org/dyn/closer.cgi/zookeeper/" target="_blank" rel="noopener">Download package from here</a></p>
<figure class="highlight shell"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br></pre></td><td class="code"><pre><span class="line">tar -zxvf zookeeper-3.4.12.tar.gz -C /work</span><br><span class="line">mv zookeeper-3.4.12 zookeeper</span><br><span class="line"></span><br><span class="line">cp conf/zoo_sample.cfg conf/zoo.cfg</span><br></pre></td></tr></table></figure>

<p>vim conf/zoo.cfg</p>
<figure class="highlight plain"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br><span class="line">9</span><br><span class="line">10</span><br><span class="line">11</span><br><span class="line">12</span><br><span class="line">13</span><br><span class="line">14</span><br></pre></td><td class="code"><pre><span class="line">tickTime&#x3D;2000</span><br><span class="line">initLimit&#x3D;10</span><br><span class="line">syncLimit&#x3D;5</span><br><span class="line">dataDir&#x3D;&#x2F;work&#x2F;zookeeper&#x2F;data</span><br><span class="line">clientPort&#x3D;2181</span><br><span class="line"></span><br><span class="line">server.0&#x3D;192.168.4.210:2888:3888</span><br><span class="line">server.1&#x3D;192.168.4.211:2888:3888</span><br><span class="line">server.2&#x3D;192.168.4.212:2888:3888</span><br><span class="line">server.3&#x3D;192.168.4.213:2888:3888</span><br><span class="line">server.4&#x3D;192.168.4.214:2888:3888</span><br><span class="line">server.5&#x3D;192.168.4.215:2888:3888</span><br><span class="line">server.6&#x3D;192.168.4.216:2888:3888</span><br><span class="line">server.7&#x3D;192.168.4.217:2888:3888</span><br></pre></td></tr></table></figure>

<p>bin/init_myid.sh, update <code>ens3</code> to your network interface</p>
<figure class="highlight shell"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br></pre></td><td class="code"><pre><span class="line"><span class="meta">#</span><span class="bash">!/bin/bash</span></span><br><span class="line">NODE_LOCAL_IP=$(ifconfig ens3 | grep "inet addr:" | awk '&#123;print $2&#125;' | cut -c 6-)</span><br><span class="line">echo $&#123;NODE_LOCAL_IP##*.&#125; &gt; /work/zookeeper/data/myid</span><br></pre></td></tr></table></figure>

<p>init cluster myid</p>
<figure class="highlight shell"><table><tr><td class="gutter"><pre><span class="line">1</span><br></pre></td><td class="code"><pre><span class="line">pssh -h /work/hosts "/work/zookeeper/bin/init_myid.sh"</span><br></pre></td></tr></table></figure>

<p>start zookeeper cluster</p>
<figure class="highlight shell"><table><tr><td class="gutter"><pre><span class="line">1</span><br></pre></td><td class="code"><pre><span class="line">pssh -h /work/hhosts -o out "/work/zookeeper/bin/zkServer.sh start"</span><br></pre></td></tr></table></figure>

<h2 id="参考内容"><a href="#参考内容" class="headerlink" title="参考内容"></a>参考内容</h2><ul>
<li><p><a href="https://code.google.com/archive/p/parallel-ssh/downloads" target="_blank" rel="noopener">parallel-ssh download page</a></p>
</li>
<li><p><a href="https://www.iteblog.com/archives/2270.html" target="_blank" rel="noopener">Hadoop 2.x Port &amp; 3.x 多个服务的默认端口被改变</a></p>
</li>
<li><p><a href="https://hadoop.apache.org/docs/r3.1.0/hadoop-project-dist/hadoop-hdfs/hdfs-default.xml" target="_blank" rel="noopener">hadoop r3.1.0 hdfs-default.xml</a></p>
</li>
<li><p><a href="https://stackoverflow.com/questions/47310619/cant-delete-hdfs-directory-via-web-interface-because-im-dr-who" target="_blank" rel="noopener">Can’t Delete HDFS Directory Via Web Interface Because I’m Dr. Who</a></p>
</li>
<li><p><a href="https://github.com/apache/spark/pull/11490/commits/d64ade1939e51bfa4da0958e1057164b725ad7f3" target="_blank" rel="noopener">[SPARK-13117] [Web UI] WebUI should use the local ip not 0.0.0.0 #11490</a></p>
</li>
</ul>

    </div>

    
    
    

      <footer class="post-footer">
          <div class="post-tags">
              <a href="/tags/spark/" rel="tag"># spark</a>
              <a href="/tags/hadoop/" rel="tag"># hadoop</a>
              <a href="/tags/zookeeper/" rel="tag"># zookeeper</a>
          </div>

        


        
    <div class="post-nav">
      <div class="post-nav-item">
    <a href="/2018/02/28/ssh-login-with-key-fail-and-require-password/" rel="prev" title="用户目录权限问题导致 SSH 免密码登录失败">
      <i class="fa fa-chevron-left"></i> 用户目录权限问题导致 SSH 免密码登录失败
    </a></div>
      <div class="post-nav-item">
    <a href="/2019/11/20/opentsdb-query-optimization/" rel="next" title="OpenTSDB 查询优化">
      OpenTSDB 查询优化 <i class="fa fa-chevron-right"></i>
    </a></div>
    </div>
      </footer>
    
  </article>
  
  
  

  </div>


          </div>
          

<script>
  window.addEventListener('tabs:register', () => {
    let activeClass = CONFIG.comments.activeClass;
    if (CONFIG.comments.storage) {
      activeClass = localStorage.getItem('comments_active') || activeClass;
    }
    if (activeClass) {
      let activeTab = document.querySelector(`a[href="#comment-${activeClass}"]`);
      if (activeTab) {
        activeTab.click();
      }
    }
  });
  if (CONFIG.comments.storage) {
    window.addEventListener('tabs:click', event => {
      if (!event.target.matches('.tabs-comment .tab-content .tab-pane')) return;
      let commentClass = event.target.classList[1];
      localStorage.setItem('comments_active', commentClass);
    });
  }
</script>

        </div>
          
  
  <div class="toggle sidebar-toggle">
    <span class="toggle-line toggle-line-first"></span>
    <span class="toggle-line toggle-line-middle"></span>
    <span class="toggle-line toggle-line-last"></span>
  </div>

  <aside class="sidebar">
    <div class="sidebar-inner">

      <ul class="sidebar-nav motion-element">
        <li class="sidebar-nav-toc">
          文章目录
        </li>
        <li class="sidebar-nav-overview">
          站点概览
        </li>
      </ul>

      <!--noindex-->
      <div class="post-toc-wrap sidebar-panel">
          <div class="post-toc motion-element"><ol class="nav"><li class="nav-item nav-level-2"><a class="nav-link" href="#Install-pssh-to-batch-config-spark-slaves"><span class="nav-number">1.</span> <span class="nav-text">Install pssh, to batch config spark slaves</span></a></li><li class="nav-item nav-level-2"><a class="nav-link" href="#System-config-in-master"><span class="nav-number">2.</span> <span class="nav-text">System config in master</span></a></li><li class="nav-item nav-level-2"><a class="nav-link" href="#Init-master-and-slaves-in-master"><span class="nav-number">3.</span> <span class="nav-text">Init master and slaves in master</span></a></li><li class="nav-item nav-level-2"><a class="nav-link" href="#Hadoop-config-in-master"><span class="nav-number">4.</span> <span class="nav-text">Hadoop config in master</span></a></li><li class="nav-item nav-level-2"><a class="nav-link" href="#Exception-solutions"><span class="nav-number">5.</span> <span class="nav-text">Exception solutions</span></a></li><li class="nav-item nav-level-2"><a class="nav-link" href="#Zookeeper-cluster-init"><span class="nav-number">6.</span> <span class="nav-text">Zookeeper cluster init</span></a></li><li class="nav-item nav-level-2"><a class="nav-link" href="#参考内容"><span class="nav-number">7.</span> <span class="nav-text">参考内容</span></a></li></ol></div>
      </div>
      <!--/noindex-->

      <div class="site-overview-wrap sidebar-panel">
        <div class="site-author motion-element" itemprop="author" itemscope itemtype="http://schema.org/Person">
    <img class="site-author-image" itemprop="image" alt="laudukang"
      src="/images/head.png">
  <p class="site-author-name" itemprop="name">laudukang</p>
  <div class="site-description" itemprop="description">Bug Not Found</div>
</div>
<div class="site-state-wrap motion-element">
  <nav class="site-state">
      <div class="site-state-item site-state-posts">
          <a href="/archives/">
        
          <span class="site-state-item-count">68</span>
          <span class="site-state-item-name">日志</span>
        </a>
      </div>
      <div class="site-state-item site-state-categories">
            <a href="/categories/">
          
        <span class="site-state-item-count">10</span>
        <span class="site-state-item-name">分类</span></a>
      </div>
      <div class="site-state-item site-state-tags">
            <a href="/tags/">
          
        <span class="site-state-item-count">25</span>
        <span class="site-state-item-name">标签</span></a>
      </div>
  </nav>
</div>
  <div class="links-of-author motion-element">
      <span class="links-of-author-item">
        <a href="https://github.com/laudukang" title="GitHub → https:&#x2F;&#x2F;github.com&#x2F;laudukang" rel="noopener" target="_blank"><i class="fa fa-fw fa-github"></i>GitHub</a>
      </span>
      <span class="links-of-author-item">
        <a href="https://gitee.com/laudukang" title="Gitee → https:&#x2F;&#x2F;gitee.com&#x2F;laudukang" rel="noopener" target="_blank"><i class="fa fa-fw fa-github-alt"></i>Gitee</a>
      </span>
      <span class="links-of-author-item">
        <a href="https://stackoverflow.com/users/5621049/laudukang" title="StackOverflow → https:&#x2F;&#x2F;stackoverflow.com&#x2F;users&#x2F;5621049&#x2F;laudukang" rel="noopener" target="_blank"><i class="fa fa-fw fa-stack-overflow"></i>StackOverflow</a>
      </span>
      <span class="links-of-author-item">
        <a href="https://www.google.com/maps/place/%E6%B7%B1%E5%9C%B3%E5%B8%82%E8%BD%AF%E4%BB%B6%E4%BA%A7%E4%B8%9A%E5%9F%BA%E5%9C%B0/@22.5244896,113.9382973,17z/data=!3m1!4b1!4m5!3m4!1s0x3403ee17d008a019:0xbb7b06b73d856c14!8m2!3d22.5244896!4d113.9382973" title="Map → https:&#x2F;&#x2F;www.google.com&#x2F;maps&#x2F;place&#x2F;%E6%B7%B1%E5%9C%B3%E5%B8%82%E8%BD%AF%E4%BB%B6%E4%BA%A7%E4%B8%9A%E5%9F%BA%E5%9C%B0&#x2F;@22.5244896,113.9382973,17z&#x2F;data&#x3D;!3m1!4b1!4m5!3m4!1s0x3403ee17d008a019:0xbb7b06b73d856c14!8m2!3d22.5244896!4d113.9382973" rel="noopener" target="_blank"><i class="fa fa-fw fa-map-marker"></i>Map</a>
      </span>
  </div>



      </div>

    </div>
  </aside>
  <div id="sidebar-dimmer"></div>


      </div>
    </main>

    <footer class="footer">
      <div class="footer-inner">
        

<div class="copyright">
  
  &copy; 2015 – 
  <span itemprop="copyrightYear">2025</span>
  <span class="with-love">
    <i class="fa fa-heart icon-next-heart"></i>
  </span>
  <span class="author" itemprop="copyrightHolder">laudukang</span>
</div>
  <div class="powered-by">由 <a href="https://hexo.io/" class="theme-link" rel="noopener" target="_blank">Hexo</a> 强力驱动 v4.2.0
  </div>
  <span class="post-meta-divider">|</span>
  <div class="theme-info">主题 – <a href="https://muse.theme-next.org/" class="theme-link" rel="noopener" target="_blank">NexT.Muse</a> v7.7.1
  </div>

        
<div class="busuanzi-count">
  <script pjax async src="https://busuanzi.ibruce.info/busuanzi/2.3/busuanzi.pure.mini.js"></script>
</div>






  <script pjax>
  function leancloudSelector(url) {
    url = encodeURI(url);
    return document.getElementById(url).querySelector('.leancloud-visitors-count');
  }
  if (CONFIG.page.isPost) {
    function addCount(Counter) {
      var visitors = document.querySelector('.leancloud_visitors');
      var url = decodeURI(visitors.getAttribute('id'));
      var title = visitors.getAttribute('data-flag-title');

      Counter('get', `/classes/Counter?where=${JSON.stringify({ url })}`)
        .then(response => response.json())
        .then(({ results }) => {
          if (results.length > 0) {
            var counter = results[0];
              leancloudSelector(url).innerText = counter.time + 1;
            Counter('put', '/classes/Counter/' + counter.objectId, { time: { '__op': 'Increment', 'amount': 1 } })
              .then(response => response.json())
              .catch(error => {
                console.error('Failed to save visitor count', error);
              })
          } else {
              Counter('post', '/classes/Counter', { title: title, url: url, time: 1 })
                .then(response => response.json())
                .then(() => {
                  leancloudSelector(url).innerText = 1;
                })
                .catch(error => {
                  console.error('Failed to create', error);
                });
          }
        })
        .catch(error => {
          console.error('LeanCloud Counter Error', error);
        });
    }
  } else {
    function showTime(Counter) {
      var visitors = document.querySelectorAll('.leancloud_visitors');
      var entries = [...visitors].map(element => {
        return decodeURI(element.getAttribute('id'));
      });

      Counter('get', `/classes/Counter?where=${JSON.stringify({ url: { '$in': entries } })}`)
        .then(response => response.json())
        .then(({ results }) => {
          if (results.length === 0) {
            document.querySelectorAll('.leancloud_visitors .leancloud-visitors-count').forEach(element => {
              element.innerText = 0;
            });
            return;
          }
          for (let item of results) {
            let { url, time } = item;
            leancloudSelector(url).innerText = time;
          }
          for (let url of entries) {
            var element = leancloudSelector(url);
            if (element.innerText == '') {
              element.innerText = 0;
            }
          }
        })
        .catch(error => {
          console.error('LeanCloud Counter Error', error);
        });
    }
  }

  fetch('https://app-router.leancloud.cn/2/route?appId=CPfRDhOlTyfkHW155W3Kx3H1-gzGzoHsz')
    .then(response => response.json())
    .then(({ api_server }) => {
      var Counter = (method, url, data) => {
        return fetch(`https://${api_server}/1.1${url}`, {
          method: method,
          headers: {
            'X-LC-Id': 'CPfRDhOlTyfkHW155W3Kx3H1-gzGzoHsz',
            'X-LC-Key': 'TX05RQSQMd8fbL7RaJNADS7S',
            'Content-Type': 'application/json',
          },
          body: JSON.stringify(data)
        });
      };
      if (CONFIG.page.isPost) {
        if (CONFIG.hostname !== location.hostname) return;
        addCount(Counter);
      } else if (document.querySelectorAll('.post-title-link').length >= 1) {
        showTime(Counter);
      }
    });
  </script>


      </div>
    </footer>
  </div>

  
  <script src="/lib/anime.min.js"></script>
  <script src="/lib/pjax/pjax.min.js"></script>
  <script src="//cdn.jsdelivr.net/npm/jquery@3/dist/jquery.min.js"></script>
  <script src="//cdn.jsdelivr.net/gh/fancyapps/fancybox@3/dist/jquery.fancybox.min.js"></script>
  <script src="/lib/velocity/velocity.min.js"></script>
  <script src="/lib/velocity/velocity.ui.min.js"></script>

<script src="/js/utils.js"></script>

<script src="/js/motion.js"></script>


<script src="/js/schemes/muse.js"></script>


<script src="/js/next-boot.js"></script>

  <script>
var pjax = new Pjax({
  selectors: [
    'head title',
    '#page-configurations',
    '.content-wrap',
    '.post-toc-wrap',
    '#pjax'
  ],
  switches: {
    '.post-toc-wrap': Pjax.switches.innerHTML
  },
  analytics: false,
  cacheBust: false,
  scrollTo : !CONFIG.bookmark.enable
});

window.addEventListener('pjax:success', () => {
  document.querySelectorAll('script[pjax], script#page-configurations, #pjax script').forEach(element => {
    var code = element.text || element.textContent || element.innerHTML || '';
    var parent = element.parentNode;
    parent.removeChild(element);
    var script = document.createElement('script');
    if (element.id) {
      script.id = element.id;
    }
    if (element.className) {
      script.className = element.className;
    }
    if (element.type) {
      script.type = element.type;
    }
    if (element.src) {
      script.src = element.src;
      // Force synchronous loading of peripheral JS.
      script.async = false;
    }
    if (element.getAttribute('pjax') !== null) {
      script.setAttribute('pjax', '');
    }
    if (code !== '') {
      script.appendChild(document.createTextNode(code));
    }
    parent.appendChild(script);
  });
  NexT.boot.refresh();
  // Define Motion Sequence & Bootstrap Motion.
  if (CONFIG.motion.enable) {
    NexT.motion.integrator
      .init()
      .add(NexT.motion.middleWares.subMenu)
      .add(NexT.motion.middleWares.postList)
      .bootstrap();
  }
  NexT.utils.updateSidebarPosition();
});
</script>




  















    <div id="pjax">
  

  
<script>window.va = window.va || function () { (window.vaq = window.vaq || []).push(arguments); };</script><script defer src="/_vercel/insights/script.js" data-endpoint="/this-is-vercel-insights"></script><script>window.si = window.si || function () { (window.siq = window.siq || []).push(arguments); };</script><script defer src="/_vercel/speed-insights/script.js" data-endpoint="/this-is-vercel-insights-speed"></script>
    </div>
</body>
</html>
